package com.g2room.driving.grap.life;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.life.model.po.ComCompany;
import com.g2room.driving.grap.life.model.po.ComShop;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.util.DbUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class Grap11467Data implements Runnable {
//http://chongqing.11467.com/
	public String baseUrl = "http://chongqing.11467.com/";
	public Map<String, ComCompany> comMap = new HashMap<String, ComCompany>();
	public Map<String, ComCompany> comExistMap = new HashMap<String, ComCompany>();
	private String url;
	private String kind;
	private String prov;
	public static void main(String[] args) throws Exception{
		
		Map<String, String> grapUrls = new HashMap<String, String>();
		grapUrls.put("http://chongqing.11467.com/", "重庆");//29766
//		grapUrls.put("http://www.dianping.com/search/category/9/85", "医疗健康");//16110
//		grapUrls.put("http://www.dianping.com/search/category/9/35", "周边游");//2707
//		grapUrls.put("http://www.dianping.com/search/category/9/80", "生活服务");//63082

//		map.put(el.attr("href"), replaceStr(text, prov));
		for(String url : grapUrls.keySet()) {
			Map<String, String> map = getCompanyKind(url, grapUrls.get(url));
			for(String key : map.keySet()) {
				Grap11467Data gkd = new Grap11467Data();
				gkd.url = key;
				gkd.kind = map.get(key);
				gkd.prov=grapUrls.get(url);
				Thread t = new Thread(gkd);
				t.start();
			}
		}
		
//		grapErrShop();
    }


	@Override
	public void run() {
		getCategory(url, kind, prov);
		System.out.println(kind+ "==================抓取完毕==================" + comExistMap.size());
		updateCom(comExistMap);
		System.out.println(kind+ "==================更新存在完毕==================" + prov);
	}
	
	public static void grapErrShop() {
		List<GrapErr> errs = getErrs();
		GrabDzdpShopDataByProxy gcd = new GrabDzdpShopDataByProxy();
		for(GrapErr ge : errs) {
			String type = ge.getType();
			try{
//				gcd.code = url.substring(url.lastIndexOf("//")+2, url.indexOf("."));
				if(type.equals("23")) {
					ComShop cs = JSON.parseObject(ge.getContent(), ComShop.class);
					gcd.getPage(cs);
				} else if(type.equals("25")) {
					ComShop cs = JSON.parseObject(ge.getContent(), ComShop.class);
					gcd.getDetail(cs);
				} else if(type.equals("14")) {
					ComShop cs = JSON.parseObject(ge.getContent(), ComShop.class);
					gcd.insert2ShopDb(cs);
				} else if(type.equals("27")) {
					ComShop cs = JSON.parseObject(ge.getContent(), ComShop.class);
					gcd.getDetail(cs);
				} 
				System.out.println(ge.getUrl());
//				delErr(ge.getId());
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public static Map<String, String> getCompanyKind(String url, String prov) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("31");
			ge.setContent(prov);
			ge.setDescription("获取Area错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		
		if(null == doc) return null;
		Elements els = doc.getElementsByAttributeValue("class", "sort");
		if(null == els || els.isEmpty())  {
			System.out.println("获取类别出错");
			return null;
		}
		els = els.first().getElementsByTag("a");
		Map<String, String> map = new HashMap<String, String>();
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				String text = el.text();
				if(text.contains("浏览")) continue;
				map.put(el.attr("href"), replaceStr(text, prov));
			}
		}
		return map;
//		for(String listUrl : map.keySet()) {
//			getCategory(listUrl, map.get(listUrl), prov);
//		}
	}
	
	private static String replaceStr(String str, String prov) {
		return null == str ? null : str.replace("公司", "").replace(prov, "");
	}
	/**
	 * 获取休闲娱乐导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getCategory(String url, String kind, String prov) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("32");
			ge.setContent(kind+","+prov);
			ge.setDescription("获取Category错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		
		if(null == doc) return;
		
		Elements els = doc.getElementsByAttributeValue("class", "box sidesubcat");
		Map<String, String> map = new HashMap<String, String>();
		if(null != els && ! els.isEmpty())  {
			els = els.first().getElementsByTag("a");
			for(Element el : els) {
				map.put(el.attr("href"), replaceStr(el.text(), prov));
			}
		}
		
		for(String typeUrl : map.keySet()) {
			getType(typeUrl, kind, map.get(typeUrl), prov);
		}
	}
	

	public void getType(String url, String kind, String cat, String prov) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("33");
			ge.setContent(kind+","+prov+","+cat);
			ge.setDescription("获取Type错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		
		if(null == doc) return;
		
		Elements els = doc.getElementsByAttributeValue("class", "box sidesubcat");
		Map<String, String> map = new HashMap<String, String>();
		if(null != els && ! els.isEmpty())  {
			els = els.first().getElementsByTag("a");
			for(Element el : els) {
				map.put(el.attr("href").replace(".htm", "-900.htm"), replaceStr(el.text(), prov));
			}
		}
		for(String pageUrl:map.keySet()) {
			getPage(pageUrl, kind, cat, map.get(pageUrl), prov);
		}
	}

	public void getPage(String url, String kind, String cat, String type, String prov) {
//		System.out.println("page=" + url + "==" + kind+","+prov+","+cat+","+type);
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("34");
			ge.setContent(kind+","+prov+","+cat+","+type);
			ge.setDescription("获取page 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		
		List<String> pageUrls = new ArrayList<String>();
		try {
			pageUrls.add(url);
			Elements els = doc.getElementsByAttributeValue("class","pages");
			if(null != els && !els.isEmpty()) {
				els = els.first().getElementsByTag("a");
				Element el = els.get(els.size() - 1);
				if(null != el) {
					int pn = Integer.valueOf(el.text());
					for(int i = 2; i<pn+1; i++) {
						pageUrls.add(url.replace("900.htm", i+".htm"));
					}
				}
			}
		} catch (Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("35");
			ge.setContent(kind+","+prov+","+cat+","+type);
			ge.setDescription("获取page 内容错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		
		for(String pageUrl : pageUrls) {
			getList(pageUrl, kind, cat, type, prov);
		}
	}

	public void getList(String url, String kind, String cat, String type, String prov) {
//		System.out.println("list=" + url + "==" + kind+","+prov+","+cat+","+type);
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("36");
			ge.setContent(kind+","+prov+","+cat+","+type);
			ge.setDescription("获取List 错误："+e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		List<ComCompany> ccs = new ArrayList<ComCompany>();
		Element el = doc.getElementById("slist");
		if(null == el) return;
		Elements els = el.getElementsByTag("li");
		if(null != els && !els.isEmpty()) {
			for(Element liel : els) {
				ComCompany cc = new ComCompany();
				try {
					cc.setKindName(kind);
					cc.setCategoryName(cat);
					cc.setTypeName(type);
					cc.setProvinceCode(prov);
					Elements aels = liel.getElementsByTag("a");
					if(null == aels || aels.isEmpty()) continue;
					cc.setSourceUrl(aels.first().attr("href"));
					cc.setCompanyName(aels.first().text());
					
//					aels = liel.getElementsByTag("dd");
//					if(null == aels || aels.isEmpty() || aels.size() <2) {
//						System.out.println("获取List 内容错误:" + kind+","+prov+","+cat+","+type + "===" + liel.html());
//						continue;
//					}
//					cc.setContactor(aels.get(1).text());
					ccs.add(cc);
				} catch (Exception e) {
					GrapErr ge = new  GrapErr();
					ge.setUrl(url);
					ge.setType("37");
					ge.setContent(kind+","+prov+","+cat+","+type);
					ge.setDescription("获取List 内容错误："+e.getMessage());
					insert2ErrDb(ge);
				}
			}
		}
		List<ComCompany> dbccs = new ArrayList<ComCompany>();
		for(ComCompany cc : ccs) {
			getExist(cc, dbccs);
		}
		
		insert2ComDb(dbccs);
	}
	
	private synchronized void getExist(ComCompany cc, List<ComCompany> dbccs) {
		ComCompany comcom = comMap.get(cc.getCompanyName());
		if(null == comcom) {
			cc = getDetail(cc);
			if(null != cc) {
				dbccs.add(cc);
				comMap.put(cc.getCompanyName(), cc);
			}
		} else {
			if(!cc.getTypeName().equals(comcom.getTypeName())) {
				cc.setTypeName(cc.getTypeName() + ";" +comcom.getTypeName());
			}
			if(!cc.getCategoryName().equals(comcom.getCategoryName())) {
				cc.setCategoryName(cc.getCategoryName() + ";" +comcom.getCategoryName());
			}
			if(!cc.getKindName().equals(comcom.getKindName())) {
				cc.setKindName(cc.getKindName() + ";" +comcom.getKindName());
			}
			
			// 存在则更新类别
			comExistMap.put(cc.getCompanyName(), cc);
		}
	}
	
	public ComCompany getDetail(ComCompany cc) {
		String url = cc.getSourceUrl();
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch(Exception e) {
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("38");
			ge.setContent(JSON.toJSONString(cc));
			ge.setDescription("获取Detail 错误："+e.getMessage());
			insert2ErrDb(ge);
			return null;
		}
		try {
			Elements els = doc.getElementsByAttributeValue("class","boxcontent text");
			if(null != els && !els.isEmpty()) {
				cc.setDescription(HtmlUtil.delATag(els.first().html()));
			}
			
			Element el = doc.getElementById("contact");
			if(null == el) return  null;
			els = el.getElementsByTag("dt");
			Elements ddels = el.getElementsByTag("dd");
			if(null != els && null != ddels && ddels.size() == els.size()) {
				for(int i = 0; i<els.size(); i++) {
					el = els.get(i);
					if(null ==el) continue;
					String val = ddels.get(i).text();
					if(el.text().contains("主营产品") || el.text().contains("主要生产")) {
						cc.setBusinessScope(val);
					} else if(el.text().contains("日期")) {
						cc.setCreateDate(val);
					} else if(el.text().contains("地址")) {
						cc.setAddress(val);
					} else if(el.text().contains("电话")) {
						cc.setPhone(val);
					} else if(el.text().contains("手机")) {
						cc.setMobil(val.replace("查看", "").trim());
					} else if(el.text().contains("邮政编码")) {
						cc.setZipCode(val);
					} else if(el.text().contains("注册资金")) {
						cc.setRegCapital(val);
					} else if(el.text().contains("网站")) {
						cc.setWebsite(val);
					} else if(el.text().contains("信用等级")) {
						el = ddels.get(i);
						if(null != el && null != el.getElementsByTag("img")) {
							cc.setStar(el.getElementsByTag("img").attr("alt"));
						}
					} else if(el.text().contains("员工数量")) {
						cc.setEmployees(val);
					} else if(el.text().contains("传真")) {
						cc.setFax(val);
					} else if(el.text().contains("经理") || el.text().contains("厂长") || el.text().contains("老板") 
							|| el.text().contains("联系人")|| el.text().contains("所长")) {
						cc.setContactor(val);
					}
				}
			}
			if(null != cc.getCompanyName() && (cc.getCompanyName().contains("个体经营") || cc.getCompanyName().contains("个人"))) {
				cc.setName(cc.getCompanyName().replace("（个体经营）", "").replace("（个人）", "").replace("(个体经营)", "").replace("(个人)", ""));
				cc.setCompanyType("个体");
			}
			return cc;
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("39");
			ge.setContent(JSON.toJSONString(cc));
			ge.setDescription("获取Detail 内容错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		return null;
	}
	
	public void insert2ComDb(List<ComCompany> ccs) {
		String sql = "insert into com_company_11467(name,create_date,source_url,contactor,description,kind_name,zip_code,reg_num,province_code,fax,type_code,principal,address,mobil,status,source,city_name,company_type,star,last_update,company_name,category_name,reg_capital,type_name,business_scope,employees,phone,area,website) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getComConn().prepareStatement(sql);
			for(ComCompany cc : ccs) {
				ps.setString(1, cc.getName());
				ps.setString(2, cc.getCreateDate());
				ps.setString(3, cc.getSourceUrl());
				ps.setString(4, cc.getContactor());
				ps.setString(5, cc.getDescription());
				ps.setString(6, cc.getKindName());
				ps.setString(7, cc.getZipCode());
				ps.setString(8, cc.getRegNum());
				ps.setString(9, cc.getProvinceCode());
				ps.setString(10, cc.getFax());
				ps.setString(11, cc.getTypeCode());
				ps.setString(12, cc.getPrincipal());
				ps.setString(13, cc.getAddress());
				ps.setString(14, cc.getMobil());
				ps.setString(15, cc.getStatus());
				ps.setString(16, cc.getSource());
				ps.setString(17, cc.getCityName());
				ps.setString(18, cc.getCompanyType());
				ps.setString(19, cc.getStar());
				ps.setString(20, cc.getLastUpdate());
				ps.setString(21, cc.getCompanyName());
				ps.setString(22, cc.getCategoryName());
				ps.setString(23, cc.getRegCapital());
				ps.setString(24, cc.getTypeName());
				ps.setString(25, cc.getBusinessScope());
				ps.setString(26, cc.getEmployees());
				ps.setString(27, cc.getPhone());
				ps.setString(28, cc.getArea());
				ps.setString(29, cc.getWebsite());
				ps.addBatch();
			}
			ps.executeBatch();	
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("db");
			ge.setContent(JSON.toJSONString(ccs));
			ge.setDescription("插入db错误："+e.getMessage());
			insert2ErrDb(ge);
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	
	public static void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getComConn().prepareStatement("insert into grap_err_11467(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	
	public static List<GrapErr> getErrs() {
		List<GrapErr> univs = new ArrayList<GrapErr>();
		try {
			Connection connection = DbUtil.getComConn();
			Statement statement = connection.createStatement();
			String sql = "select * from grap_err_11467";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				GrapErr univ = new GrapErr();
				univ.setId(rs.getInt("id"));
				univ.setUrl(rs.getString("url"));
				univ.setType(rs.getString("type"));
				univ.setContent(rs.getString("content"));
				univs.add(univ);
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return univs;
	}
	
	public static void delErr(int id) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getComConn().prepareStatement("delete from grap_err_11467 where id=?");
			grapPs.setInt(1, id);
			grapPs.execute();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					
				}
			}
		}
	}
	
	public void updateCom(Map<String, ComCompany> map) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getComConn().prepareStatement("insert into com_exist(type_name,category_name,kind_name,company_name) values(?,?,?,?)");
			for(String cn : map.keySet()) {
				ComCompany com= map.get(cn);
				grapPs.setString(1, com.getTypeName());
				grapPs.setString(2, com.getCategoryName());
				grapPs.setString(3, com.getKindName());
				grapPs.setString(4, cn);
				grapPs.addBatch();
			}
			grapPs.executeBatch();
		} catch(Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("db1");
			ge.setContent(JSON.toJSONString(map));
			ge.setDescription("插入db错误："+e.getMessage());
			insert2ErrDb(ge);
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					
				}
			}
		}
	}
}